'''
分析训练过程中的生成的log日志
'''
import os
import pandas as pds
import numpy as np
from matplotlib import pyplot as plt
from collections import Counter
import matplotlib.colors as mcolors
import random
# 解析csv文件

def readlog(csvpath):
    datals={"epoch":[],"loss":[],"acc":[],"lb":[],"plb":[],"imnames":[]}
    with open(csvpath,'r',encoding='utf-8') as fp:
        while True:
            line=fp.readline()
            if line is None or line=="":
                break
            # 解析文件
            ls=line.split(",")
            datals["epoch"].append(ls[0])
            datals["loss"].append(ls[1])
            datals["acc"].append(ls[2])
            datals["lb"].append(ls[3])
            datals["plb"].append(ls[4])
            datals["imnames"].append(ls[5])
    return  pds.DataFrame(datals)


def logarreshape(lossarr):
    # 计算每一个批次的信息
    epoch=dict(Counter(lossarr[:,0].tolist()))
    step=epoch[list(epoch.keys())[0]]
    print("step:{}".format(step))
    # loss
    lossarr=lossarr.reshape(-1,step,3)
    return lossarr    

def outCorrLtZero(trainloss,testloss,vailloss,start,end,isdraw=True):
    # 平均损失
    trainmeanloss=np.mean(trainloss,axis=1)
    testmeanloss=np.mean(testloss,axis=1)
    vailmeanloss=np.mean(vailloss,axis=1)
    if isdraw:
        # 绘制
        plt.figure()
        plt.plot(trainmeanloss[start:end,0], trainmeanloss[start:end,1], '-', marker="*",label='trainmean', linewidth=1,color="green")
        plt.plot(testmeanloss[start:end,0], testmeanloss[start:end,1], '-.', marker=".", label='testmean', linewidth=1,color="red")
        plt.plot(vailmeanloss[start:end,0], vailmeanloss[start:end,1], '--', marker="+", label='vailmean', linewidth=1,color="blue")    
        plt.legend()
        plt.title("train-vail-test")
        plt.show()

    tt_vl=np.corrcoef(testmeanloss[start:end,1],vailmeanloss[start:end,1])
    tr_tt=np.corrcoef(trainmeanloss[start:end,1],testmeanloss[start:end,1])
    print("验证集-测试集",tt_vl[0,1]) # 相关系数  0.99066198 
    print("训练集-测试集",tr_tt[0,1]) # 相关系数  -0.8333207186225663 

    # 训练集中，单个样本的训练情况与 整体情况的 相关系数分析
    corr_tr_tt=np.zeros((trainloss.shape[1],2))
    for i in range(trainloss.shape[1]):
        corr_tr_tt[i,0]=i
        corr_tr_tt[i,1]=np.corrcoef(trainloss[start:end,i,1],testmeanloss[start:end,1])[0,1]

    print("{}_{}  相关系数平均值".format(start,end),np.mean(corr_tr_tt[:,1]))
    
    if isdraw:
        # 绘制图像 整体相关系数 分布情况
        plt.figure()
        plt.scatter(corr_tr_tt[:,0], corr_tr_tt[:,1], marker="*",label='tr_tt', linewidth=1,color="green")
        plt.show()

    # 选择 相关系数 最低的,100 ---通过排序实现
    newidx=np.argsort(corr_tr_tt[:,1],axis=0)
    new_corr=corr_tr_tt[newidx,:]
    idx=np.where(new_corr[:,1]<0)

    print(idx[0].shape)
    # 绘制这些数据的相关系数的分布
    LT_zero=new_corr[idx,:]
    colors=list(mcolors.cnames.keys())
    # 绘制图像 整体相关系数 分布情况
    if isdraw:
        plt.figure()
        plt.hist(LT_zero[0,:,1])
        plt.title("{}_{}_LtZero_hist".format(start,end))
        plt.show()
        plt.figure()
        # 抽取其中 展示信息
        t=0
        for i in idx[0].tolist()[:20]:
            colorname=colors[i+t]
            if colorname=='red':
                t=t+1
                colorname=colors[i+t]
            maxtrloss_x=trainloss[start:end,int(new_corr[i,0]),0]
            maxtrloss_y=trainloss[start:end,int(new_corr[i,0]),1]
            plt.plot(maxtrloss_x, maxtrloss_y, '-', marker="*",label='tr', linewidth=1,color=mcolors.cnames[colorname])
        plt.plot(testmeanloss[start:end,0], testmeanloss[0:end,1], '--', marker="+",label='tt', linewidth=1,color="red")
        plt.title("{} {} part ".format(start,end))
        plt.show()
    return set(LT_zero[0,:,0].astype(np.int).tolist())

def outCorrOfTrainAndTest(traindelta,testdelta,vaildelta,start,end,isdraw=True):
    # 求解增量之间 相关性
    dtt=testdelta[:,1]
    dvl=vaildelta[:,1]
    ttAndVlCorr=np.corrcoef(dtt,dvl)[0,1]
    print("增量相关性 test-vail {}".format(ttAndVlCorr))
    corr_tr_tt=np.zeros((traindelta.shape[1],2))
    for i in range(traindelta.shape[1]):
        corr_tr_tt[i,0]=i
        corr_tr_tt[i,1]=np.corrcoef(traindelta[start:end,i,1],testdelta[start:end,1])[0,1]
    print("增量相关性  test-train mean {}".format(np.mean(corr_tr_tt[:,1])))
    if isdraw:
        # 绘制图像 整体相关系数 分布情况
        plt.figure()
        plt.scatter(corr_tr_tt[:,0], corr_tr_tt[:,1], marker="*",label='tr_tt', linewidth=1,color="green")
        plt.show()
    
    # 通过排序实现相关性
    newidx=np.argsort(corr_tr_tt[:,1],axis=0)
    new_corr=corr_tr_tt[newidx,:]
    idx=np.where(new_corr[:,1]<0)
    print(idx[0].shape)
    # 绘制这些数据的相关系数的分布
    LT_zero=new_corr[idx,:]
    colors=list(mcolors.cnames.keys())
    # 绘制图像 整体相关系数 分布情况
    if isdraw:
        plt.figure()
        plt.hist(LT_zero[0,:,1])
        plt.title("{}_{}_LtZero_hist".format(start,end))
        plt.show()
        plt.figure()
        # 抽取其中 展示信息
        t=0
        for i in idx[0].tolist()[:2]:
            colorname=colors[i+t]
            if colorname=='red':
                t=t+1
                colorname=colors[i+t]
            maxtrloss_x=testdelta[start:end,0]
            maxtrloss_y=traindelta[start:end,int(new_corr[i,0]),1]
            plt.plot(maxtrloss_x, maxtrloss_y, '-', marker="*",label='tr', linewidth=1,color=mcolors.cnames[colorname])
        plt.plot(testdelta[start:end,0], testdelta[0:end,1], '--', marker="+",label='tt', linewidth=1,color="red")
        plt.title("{} {} delta part ".format(start,end))
        plt.show()
    return set(LT_zero[0,:,0].astype(np.int).tolist())

# 输出训练过程中，相关系数为 0 
def logzeroCorr(logpath,trainpds,Ltzeros):
        with open(logpath,'w',encoding='utf-8') as fp:
            for i in Ltzeros:
                
                for j in trainpds.loc[i,"imnames"].replace("[","").replace("]","").split(" "):
                    if j=="":
                        continue
                    fp.write("{}\n".format(j))

# 文件路径
root="/home/gis/gisdata/data/jupyterlabhub/gitcode/cardanger/trainanays/modeltrain12_17_10_13"
logdir=os.path.join(root,"log")

trainlog=os.path.join(logdir,"train.csv")
testlog=os.path.join(logdir,"test.csv")
vaillog=os.path.join(logdir,"vail.csv")

# 开始读取文件

trainpds=readlog(trainlog)
testpds=readlog(testlog)
vailpds=readlog(vaillog)

# 选择 批次 损失值  精度
trainloss=np.array(trainpds[["epoch","loss","acc"]]).astype(np.float)
testloss=np.array(testpds[["epoch","loss","acc"]]).astype(np.float)
vailloss=np.array(vailpds[["epoch","loss","acc"]]).astype(np.float)

# loss
trainloss=logarreshape(trainloss).astype(np.float)
testloss=logarreshape(testloss).astype(np.float)
vailloss=logarreshape(vailloss).astype(np.float)

# delta of loss
traindelta=trainloss[1:-1,:,:]-trainloss[0:-2,:,:] # 前后增量
traindelta=traindelta[1:,:]

testmeanloss=np.mean(testloss,axis=1)
testdelta=testmeanloss[1:-1,:]-testmeanloss[0:-2,:]
testdelta=testdelta[:-1,...]

vailmeanloss=np.mean(vailloss,axis=1)
vaildelta=vailmeanloss[1:-1,:]-vailmeanloss[0:-2,:]
vaildelta=vaildelta[:-1,...]

indexarr=np.array(list(range(vaildelta.shape[0])))
vaildelta[:,0]=indexarr
testdelta[:,0]=indexarr



Ltzeros_0T10=outCorrOfTrainAndTest(traindelta,testdelta,vaildelta,0,400,isdraw=True)
Ltzeros_0T20=outCorrOfTrainAndTest(traindelta,testdelta,vaildelta,0,20,isdraw=True)
Ltzeros_0T25=outCorrOfTrainAndTest(traindelta,testdelta,vaildelta,0,25,isdraw=True)
Ltzeros_0T30=outCorrOfTrainAndTest(traindelta,testdelta,vaildelta,0,30,isdraw=True)



Ltzeros_0T10=outCorrLtZero(trainloss,testloss,vailloss,0,400,isdraw=True)
Ltzeros_0T20=outCorrLtZero(trainloss,testloss,vailloss,0,20,isdraw=True)
Ltzeros_0T25=outCorrLtZero(trainloss,testloss,vailloss,0,25,isdraw=True)
Ltzeros_0T30=outCorrLtZero(trainloss,testloss,vailloss,0,30,isdraw=True)
Ltzeros_0T20=outCorrLtZero(trainloss,testloss,vailloss,0,20)
# 出现了 负相关，是训练过程有数据出了问题  （考虑使用 随机初始化）
Ltzeros=Ltzeros_0T10.intersection(Ltzeros_0T20).intersection(Ltzeros_0T25).intersection(Ltzeros_0T30)
print(len(Ltzeros))
print(Ltzeros)
# 输出所有不相干数据的 信息，并存储进入对应的文件中
logzeroCorr(os.path.join(root,"errcorf.csv"),trainpds,Ltzeros)

# 首先计算损失值 变化较大的



# 绘制变换 情况
#plt.legend()
plt.show()

# 绘制线段

'''

fig=plt.figure()
for i in range(trainloss.shape[1]):
    x=trainloss[14:20,i,0]
    y=trainloss[14:20,i,1]
    plt.plot(x, y, '-', marker="*",label='train', linewidth=1,color="green")
'''
'''
for i in range(testloss.shape[1]):
    x=testloss[14:20,i,0]
    y=testloss[14:20,i,1]
    plt.plot(x, y, '-.', marker=".", label='test', linewidth=1,color="red")
for i in range(vailloss.shape[1]):
    x=vailloss[14:20,i,0]
    y=vailloss[14:20,i,1]
    plt.plot(x, y, '--', marker="+", label='vail', linewidth=1,color="blue")    
'''
'''
plt.title("train-vail-test")
plt.show()
'''
